# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from DBUtil import DBConn
from scrapy.utils.project import get_project_settings
from scrapy.exceptions import DropItem
import requests
import time

settings = get_project_settings()
conn = DBConn(
    host=settings['HOST'],
    user=settings['USER'],
    passwd=settings['PASSWD'],
    database=settings['DATABASE']
)


# 去重
class IpproxyPipelineDeldup(object):
    def process_item(self, item, spider):
        if conn.qv("select count(1) from t_agent_ip where ip='%s' and port='%s'" % (item['ip'], item['port'])) == 0:
            return item
        else:
            raise DropItem("Duplicate item found: %s" % item)


# 验证
class IpproxyPipelineValid(object):
    def process_item(self, item, spider):
        if item['anonymous'].find('匿') == -1:
            raise DropItem('非匿名代理，丢弃 %s:%s %s' % (item['ip'], item['port'], item['anonymous']))
        http_url = "http://www.baidu.com"
        proxy_url = "http://{0}:{1}".format(item['ip'], item['port'])
        try:
            proxy_dict = {
                'http': proxy_url,
            }
            response = requests.get(http_url, proxies=proxy_dict)
        except Exception:
            raise DropItem('验证失败，丢弃 %s' % item)
        else:
            code = response.status_code
            if 200 <= code < 300:
                item['use_count'] = '0'
                return item
            else:
                raise DropItem('验证失败，丢弃 %s' % item)


# 入库存储
class IpproxyPipelineToDB(object):
    def process_item(self, item, spider):
        item['createtime'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
        conn.insert('t_agent_ip', [item])
        return item
